#! /bin/env bash

# Parse proxy_list from websites
# Concatenate proxy files together
# 1. http
python ./downloader/parse_cool_proxy_list.py &
python ./downloader/parse_rosinstrument_proxy_list.py http &
python ./downloader/parse_cz_proxy.py http &
python ./downloader/parse_cnproxy_proxy.py http &
python ./downloader/parse_proxylists_proxy.py http &
wait

cat \
    ./proxy_list/http_proxy_cool.txt \
    ./proxy_list/http_proxy_rosinstrument.txt \
    ./proxy_list/http_proxy_cz.txt \
    ./proxy_list/http_proxy_cnproxy.txt \
    ./proxy_list/http_proxy_proxylists.txt \
    > ./proxy_list/http_proxy_all_temp
uniq ./proxy_list/http_proxy_all_temp > ./proxy_list/http_proxy_all.txt
rm ./proxy_list/http_proxy_all_temp

# 2.socks
# python ./downloader/parse_rosinstrument_proxy_list.py socks
# cat \
#     ./proxy_list/socks_proxy_rosinstrument.txt \
#     > ./proxy_list/socks_proxy_all_temp
# uniq ./proxy_list/socks_proxy_all_temp > ./proxy_list/socks_proxy_all.txt
# rm ./proxy_list/socks_proxy_all_temp

# 3.https
python ./downloader/parse_rosinstrument_proxy_list.py https
cat \
    ./proxy_list/https_proxy_rosinstrument.txt \
    > ./proxy_list/https_proxy_all_temp
uniq ./proxy_list/https_proxy_all_temp > ./proxy_list/https_proxy_all.txt
rm ./proxy_list/https_proxy_all_temp
